package org.example.demo;

import ch.qos.logback.core.encoder.EchoEncoder;
import com.alibaba.fastjson.JSONObject;
import org.apache.commons.logging.LogFactory;
import org.apache.http.Header;
import org.apache.http.HttpResponse;
import org.apache.http.util.EntityUtils;
import org.example.utils.CookieUtils;
import org.example.utils.CrackJavaScript;
import org.example.utils.HttpUtils;
import org.jsoup.Jsoup;
import org.jsoup.nodes.Document;
import org.jsoup.nodes.Element;
import org.jsoup.select.Elements;
import org.slf4j.LoggerFactory;

import java.util.*;
import java.util.concurrent.atomic.AtomicInteger;

public class CrawlerTest {


    public static String requestUrl = "https://www.cnvd.org.cn/flaw/typeResult?typeId=27&max=20&offset=0";


    public static String prefixRequestUrl = "https://www.cnvd.org.cn";

    private static final AtomicInteger atomicInteger = new AtomicInteger();

    public static void main(String[] args) throws Exception {

        Map<String, String> headers = new HashMap<>();
        headers.put("User-Agent", "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/136.0.0.0 Safari/537.36");
        headers.put("Accept", "text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8");
        headers.put("Upgrade-Insecure-Requests", "1");
        headers.put("Cookie", "");
        HttpResponse response = null;
        String jsluid = "", jslClearance = "";
        while (true) {
            try {
                //第一次请求
                response = HttpUtils.doGet(requestUrl, headers, null);
                if (response.getStatusLine().getStatusCode() == 521) {
                    Header[] headers1 = response.getHeaders("Set-Cookie");
                    if (headers1 != null && headers1.length > 0) {
                        String cookie = headers1[0].getValue();
                        jsluid = "__jsluid_s=" + cookie.split(";")[0].split("=")[1];
                        jslClearance = CrackJavaScript.getJslClearance(EntityUtils.toString(response.getEntity()));
                        cookie = jsluid + ";" + jslClearance;
                        headers.put("Cookie", cookie);
                    }
                }
                //第二次
                response = HttpUtils.doGet(requestUrl, headers, null);
                if (response.getStatusLine().getStatusCode() == 521) {
                    String resHtml = EntityUtils.toString(response.getEntity());
                    // 对返回js处理 拿到jsl_clearance
                    resHtml = resHtml.substring(resHtml.lastIndexOf("go(") + 3, resHtml.lastIndexOf(")"));
                    JSONObject data = JSONObject.parseObject(resHtml);

                    // resHtml
                    //<script>var _0x3c04=['wrMhGG0=','KMOnwoMR','Q3ROw7E=','w7zDjsKABQ==','wrgXPR4=','wq0SDsKT','cw/DgcKz','wpfDoEvCqA==','woQzwq46','w6PCqMKJw40=',
                    // 返回的数据需要进行解密
                    jslClearance = CookieUtils.go(data);
                    headers.put("Cookie", jsluid + ";" + jslClearance);
                }
                //第三次请求
                response = HttpUtils.doGet(requestUrl, headers, null);
                //直到请求成功 才跳出循环
                if (response.getStatusLine().getStatusCode() == 200) {
                    break;
                }
            } catch (Exception e) {
            }
        }
        String html = EntityUtils.toString(response.getEntity());
        Document document = Jsoup.parse(html);

        //获取分页页码
        Elements pagesElements = document.getElementsByClass("pages clearfix");
        Element page = pagesElements.get(0);
        Elements pageHrefs = page.getElementsByTag("a");
        Set<String> pageClick = new HashSet<>();
        String pageClickUrl = prefixRequestUrl + "/flaw/typeResult?typeId=27&max=20&offset=0";
        
        pageClick.add("/flaw/typeResult?typeId=27&max=20&offset=0");
        for (Element pageHref : pageHrefs) {
            pageClick.add(prefixRequestUrl + pageHref.attr("href"));
        }

        for (String url : pageClick) {
            System.out.println(url);
            try {
                response = HttpUtils.doGet(url, headers, null);
            } catch (Exception e) {
                break;
            }
            String pageHtml = EntityUtils.toString(response.getEntity());
            document = Jsoup.parse(pageHtml);
            Elements tbodyElements = document.getElementsByTag("tbody");
            if (tbodyElements.size() > 0) {
                Element tbody = tbodyElements.get(0);
                Elements elements = tbody.getElementsByTag("a");
                System.out.println("获取A标签");
                List<String> hrefs = new ArrayList<>();
                elements.stream().forEach(element -> {
                    hrefs.add(element.attr("href"));
                });
                //继续请求
                for (String href : hrefs) {

                    //再次判断是否请求已经达到时间
                    while (true) {
                        try {
                            response = HttpUtils.doGet(prefixRequestUrl + href, headers, null);
                            if (response.getStatusLine().getStatusCode() == 521) {
                                String resHtml = EntityUtils.toString(response.getEntity());
                                // 对返回js处理 拿到jsl_clearance
                                resHtml = resHtml.substring(resHtml.lastIndexOf("go(") + 3, resHtml.lastIndexOf(")"));
                                JSONObject data = JSONObject.parseObject(resHtml);
                                // resHtml
                                //<script>var _0x3c04=['wrMhGG0=','KMOnwoMR','Q3ROw7E=','w7zDjsKABQ==','wrgXPR4=','wq0SDsKT','cw/DgcKz','wpfDoEvCqA==','woQzwq46','w6PCqMKJw40=',
                                // 返回的数据需要进行解密
                                jslClearance = CookieUtils.go(data);
                                headers.put("Cookie", jsluid + ";" + jslClearance);
                            }
                            if (response.getStatusLine().getStatusCode() == 200) {
                                break;
                            }
                        } catch (Exception e) {

                        }

                    }
                    System.out.println(prefixRequestUrl + href + ": " + response.getStatusLine().getStatusCode());
                    html = EntityUtils.toString(response.getEntity());
                    //解析具体数据
                    document = Jsoup.parse(html);
                    Elements tables = document.getElementsByClass("gg_detail");
                    if (tables.size() > 0) {
                        Element table = tables.get(0);
                        Elements trs = table.getElementsByTag("tr");
                        for (Element tr : trs) {
                            atomicInteger.incrementAndGet();
                            Elements tds = tr.getElementsByTag("td");
                            StringBuilder stringBuilder = new StringBuilder();
                            if (tds.size() == 2) {
                                stringBuilder.append(tds.get(0).text()).append(": ").append(tds.get(1).text());
                                System.out.println(stringBuilder.toString());
                            }
                        }
                    }
                }
            }
        }
        System.out.println("========================共爬取 " + atomicInteger.get() + " 条数据======================================");
    }
}
